#pragma once

#include<string>
#include<vector>

struct Token
{
    std::string m_sValue;
	bool        m_bIsKeyword;
	bool        m_bIsDelimiter;

    Token()
    {
		m_sValue = "";
		m_bIsDelimiter = false;
		m_bIsKeyword   = false;
    }

    Token(char inChar)
    {
        m_sValue.push_back(inChar);
        //TODO Add check if really delimiter though it doesn't really matter
    }

    operator char()
    {
        if(m_sValue.length()==1)
            return m_sValue[0];
        return '\0';
    }
};

struct Tokenizer
{
public:
    std::vector<Token>      m_asTokens;
    static std::string      m_sDelimiters;
    
public:
    void tokenize(std::string inTokenStream);
    void print(void);
	void emitToken(std::string insToken);
	void tokenizeDelimiterStream(std::string inDelimiters);

    Tokenizer(void);
    ~Tokenizer(void);
};
